/*     */ package com.zimbra.cs.store.file;
/*     */ 
/*     */ import com.google.common.collect.HashMultimap;
/*     */ import com.google.common.collect.Multimap;
/*     */ import com.google.common.collect.TreeMultimap;
/*     */ import com.zimbra.common.service.ServiceException;
/*     */ import com.zimbra.common.soap.Element;
/*     */ import com.zimbra.common.util.ByteUtil;
/*     */ import com.zimbra.common.util.FileUtil;
/*     */ import com.zimbra.common.util.Log;
/*     */ import com.zimbra.common.util.ZimbraLog;
/*     */ import com.zimbra.cs.db.DbBlobConsistency;
/*     */ import com.zimbra.cs.db.DbPool;
/*     */ import com.zimbra.cs.db.DbPool.DbConnection;
/*     */ import com.zimbra.cs.mailbox.Mailbox;
/*     */ import com.zimbra.cs.mailbox.MailboxManager;
/*     */ import com.zimbra.cs.store.StoreManager;
/*     */ import com.zimbra.cs.volume.Volume;
/*     */ import com.zimbra.cs.volume.VolumeManager;
/*     */ import java.io.File;
/*     */ import java.io.FileInputStream;
/*     */ import java.io.IOException;
/*     */ import java.util.ArrayList;
/*     */ import java.util.Collection;
/*     */ import java.util.Comparator;
/*     */ import java.util.Iterator;
/*     */ import java.util.List;
/*     */ import java.util.Set;
/*     */ import java.util.regex.Matcher;
/*     */ import java.util.regex.Pattern;
/*     */ import java.util.zip.GZIPInputStream;
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ public class BlobConsistencyChecker
/*     */ {
/*     */   public static class Results
/*     */   {
/*     */     public int mboxId;
/*  73 */     public Multimap<Integer, BlobConsistencyChecker.BlobInfo> missingBlobs = TreeMultimap.create(new IntegerComparator(), new BlobInfoComparator());
/*  74 */     public Multimap<Integer, BlobConsistencyChecker.BlobInfo> incorrectSize = TreeMultimap.create(new IntegerComparator(), new BlobInfoComparator());
/*  75 */     public Multimap<Integer, BlobConsistencyChecker.BlobInfo> unexpectedBlobs = TreeMultimap.create(new IntegerComparator(), new BlobInfoComparator());
/*  76 */     public Multimap<Integer, BlobConsistencyChecker.BlobInfo> incorrectModContent = TreeMultimap.create(new IntegerComparator(), new BlobInfoComparator());
/*  77 */     public Multimap<Integer, BlobConsistencyChecker.BlobInfo> usedBlobs = TreeMultimap.create(new IntegerComparator(), new BlobInfoComparator());
/*     */     public Results() {}
/*     */     
/*     */     class IntegerComparator implements Comparator<Integer> { IntegerComparator() {}
/*     */       
/*  82 */       public int compare(Integer o1, Integer o2) { return o1.compareTo(o2); }
/*     */     }
/*     */     
/*     */     class BlobInfoComparator implements Comparator<BlobConsistencyChecker.BlobInfo> {
/*     */       BlobInfoComparator() {}
/*     */       
/*     */       public int compare(BlobConsistencyChecker.BlobInfo o1, BlobConsistencyChecker.BlobInfo o2) {
/*  89 */         return o1.path.compareTo(o2.path);
/*     */       }
/*     */     }
/*     */     
/*     */ 
/*     */ 
/*     */     public Results(Element mboxElement)
/*     */       throws ServiceException
/*     */     {
/*  98 */       if (!mboxElement.getName().equals("mbox")) {
/*  99 */         throw ServiceException.INVALID_REQUEST("Unexpected element: " + mboxElement.getName(), null);
/*     */       }
/* 101 */       this.mboxId = ((int)mboxElement.getAttributeLong("id"));
/* 102 */       for (Element item : mboxElement.getElement("missingBlobs").listElements("item")) {
/* 103 */         BlobConsistencyChecker.BlobInfo blob = new BlobConsistencyChecker.BlobInfo();
/* 104 */         blob.itemId = ((int)item.getAttributeLong("id"));
/* 105 */         blob.modContent = ((int)item.getAttributeLong("rev"));
/* 106 */         blob.dbSize = item.getAttributeLong("s");
/* 107 */         blob.volumeId = ((short)(int)item.getAttributeLong("volumeId"));
/* 108 */         blob.path = item.getAttribute("blobPath");
/* 109 */         blob.external = item.getAttributeBool("external", false);
/* 110 */         blob.version = ((int)item.getAttributeLong("version"));
/* 111 */         this.missingBlobs.put(Integer.valueOf(blob.itemId), blob);
/*     */       }
/* 113 */       for (Element itemEl : mboxElement.getElement("incorrectSize").listElements("item")) {
/* 114 */         BlobConsistencyChecker.BlobInfo blob = new BlobConsistencyChecker.BlobInfo();
/* 115 */         blob.itemId = ((int)itemEl.getAttributeLong("id"));
/* 116 */         blob.modContent = ((int)itemEl.getAttributeLong("rev"));
/* 117 */         blob.dbSize = itemEl.getAttributeLong("s");
/* 118 */         blob.volumeId = ((short)(int)itemEl.getAttributeLong("volumeId"));
/*     */         
/* 120 */         Element blobEl = itemEl.getElement("blob");
/* 121 */         blob.path = blobEl.getAttribute("path");
/* 122 */         blob.fileDataSize = Long.valueOf(blobEl.getAttributeLong("s"));
/* 123 */         blob.fileSize = Long.valueOf(blobEl.getAttributeLong("fileSize"));
/* 124 */         blob.external = blobEl.getAttributeBool("external", false);
/* 125 */         this.incorrectSize.put(Integer.valueOf(blob.itemId), blob);
/*     */       }
/* 127 */       for (Element blobEl : mboxElement.getElement("unexpectedBlobs").listElements("blob")) {
/* 128 */         BlobConsistencyChecker.BlobInfo blob = new BlobConsistencyChecker.BlobInfo();
/* 129 */         blob.volumeId = ((short)(int)blobEl.getAttributeLong("volumeId"));
/* 130 */         blob.path = blobEl.getAttribute("path");
/* 131 */         blob.fileSize = Long.valueOf(blobEl.getAttributeLong("fileSize"));
/* 132 */         blob.external = blobEl.getAttributeBool("external", false);
/* 133 */         this.unexpectedBlobs.put(Integer.valueOf(blob.itemId), blob);
/*     */       }
/* 135 */       for (Element itemEl : mboxElement.getElement("incorrectRevision").listElements("item")) {
/* 136 */         BlobConsistencyChecker.BlobInfo blob = new BlobConsistencyChecker.BlobInfo();
/* 137 */         blob.itemId = ((int)itemEl.getAttributeLong("id"));
/* 138 */         blob.modContent = ((int)itemEl.getAttributeLong("rev"));
/* 139 */         blob.dbSize = itemEl.getAttributeLong("s");
/* 140 */         blob.volumeId = ((short)(int)itemEl.getAttributeLong("volumeId"));
/*     */         
/* 142 */         Element blobEl = itemEl.getElement("blob");
/* 143 */         blob.path = blobEl.getAttribute("path");
/* 144 */         blob.fileSize = Long.valueOf(blobEl.getAttributeLong("fileSize"));
/* 145 */         blob.fileModContent = ((int)blobEl.getAttributeLong("rev"));
/* 146 */         blob.external = blobEl.getAttributeBool("external", false);
/* 147 */         this.incorrectModContent.put(Integer.valueOf(blob.itemId), blob);
/*     */       }
/* 149 */       for (Element itemEl : mboxElement.getElement("usedBlobs").listElements("item")) {
/* 150 */         BlobConsistencyChecker.BlobInfo blob = new BlobConsistencyChecker.BlobInfo();
/* 151 */         blob.itemId = ((int)itemEl.getAttributeLong("id"));
/* 152 */         blob.version = ((int)itemEl.getAttributeLong("rev"));
/* 153 */         blob.dbSize = itemEl.getAttributeLong("s");
/* 154 */         blob.volumeId = ((short)(int)itemEl.getAttributeLong("volumeId"));
/*     */         
/* 156 */         Element blobEl = itemEl.getElement("blob");
/* 157 */         blob.path = blobEl.getAttribute("path");
/* 158 */         blob.fileSize = Long.valueOf(blobEl.getAttributeLong("fileSize"));
/* 159 */         blob.external = blobEl.getAttributeBool("external", false);
/* 160 */         this.usedBlobs.put(Integer.valueOf(blob.itemId), blob);
/*     */       }
/*     */     }
/*     */     
/*     */     public boolean hasInconsistency() {
/* 165 */       return (!this.missingBlobs.isEmpty()) || (!this.incorrectSize.isEmpty()) || (!this.unexpectedBlobs.isEmpty()) || (!this.incorrectModContent.isEmpty());
/*     */     }
/*     */     
/*     */     public void toElement(Element parent)
/*     */     {
/* 170 */       Element missingEl = parent.addElement("missingBlobs");
/* 171 */       Element incorrectSizeEl = parent.addElement("incorrectSize");
/* 172 */       Element unexpectedBlobsEl = parent.addElement("unexpectedBlobs");
/* 173 */       Element incorrectRevisionEl = parent.addElement("incorrectRevision");
/* 174 */       Element usedBlobEl = parent.addElement("usedBlobs");
/*     */       
/* 176 */       for (BlobConsistencyChecker.BlobInfo blob : this.missingBlobs.values()) {
/* 177 */         missingEl.addElement("item").addAttribute("id", blob.itemId).addAttribute("rev", blob.modContent).addAttribute("s", blob.dbSize).addAttribute("volumeId", blob.volumeId).addAttribute("blobPath", blob.path).addAttribute("external", blob.external).addAttribute("version", blob.version);
/*     */       }
/*     */       
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/* 186 */       for (BlobConsistencyChecker.BlobInfo blob : this.incorrectSize.values()) {
/* 187 */         Element itemEl = incorrectSizeEl.addElement("item").addAttribute("id", blob.itemId).addAttribute("rev", blob.modContent).addAttribute("s", blob.dbSize).addAttribute("volumeId", blob.volumeId);
/*     */         
/*     */ 
/*     */ 
/*     */ 
/* 192 */         itemEl.addElement("blob").addAttribute("path", blob.path).addAttribute("s", blob.fileDataSize).addAttribute("fileSize", blob.fileSize).addAttribute("external", blob.external);
/*     */       }
/*     */       
/*     */ 
/*     */ 
/*     */ 
/* 198 */       for (BlobConsistencyChecker.BlobInfo blob : this.unexpectedBlobs.values()) {
/* 199 */         unexpectedBlobsEl.addElement("blob").addAttribute("volumeId", blob.volumeId).addAttribute("path", blob.path).addAttribute("fileSize", blob.fileSize).addAttribute("external", blob.external);
/*     */       }
/*     */       
/*     */ 
/*     */ 
/*     */ 
/* 205 */       for (BlobConsistencyChecker.BlobInfo blob : this.incorrectModContent.values()) {
/* 206 */         Element itemEl = incorrectRevisionEl.addElement("item").addAttribute("id", blob.itemId).addAttribute("rev", blob.modContent).addAttribute("s", blob.dbSize).addAttribute("volumeId", blob.volumeId);
/*     */         
/*     */ 
/*     */ 
/*     */ 
/* 211 */         itemEl.addElement("blob").addAttribute("path", blob.path).addAttribute("fileSize", blob.fileSize).addAttribute("rev", blob.fileModContent).addAttribute("external", blob.external);
/*     */       }
/*     */       
/*     */ 
/*     */ 
/*     */ 
/* 217 */       for (BlobConsistencyChecker.BlobInfo blob : this.usedBlobs.values()) {
/* 218 */         Element itemEl = usedBlobEl.addElement("item").addAttribute("id", blob.itemId).addAttribute("rev", blob.version).addAttribute("s", blob.dbSize).addAttribute("volumeId", blob.volumeId);
/*     */         
/*     */ 
/*     */ 
/*     */ 
/* 223 */         itemEl.addElement("blob").addAttribute("path", blob.path).addAttribute("s", blob.fileDataSize).addAttribute("fileSize", blob.fileSize).addAttribute("external", blob.external);
/*     */       }
/*     */     }
/*     */   }
/*     */   
/*     */ 
/*     */ 
/*     */ 
/*     */ 
/* 232 */   protected static final Log log = ZimbraLog.store;
/*     */   protected Results results;
/*     */   protected int mailboxId;
/* 235 */   protected boolean checkSize = true;
/* 236 */   protected boolean reportUsedBlobs = false;
/*     */   
/*     */ 
/*     */ 
/*     */   public Results check(Collection<Short> volumeIds, int mboxId, boolean checkSize, boolean reportUsedBlobs)
/*     */     throws ServiceException
/*     */   {
/* 243 */     StoreManager sm = StoreManager.getInstance();
/* 244 */     if (!(sm instanceof FileBlobStore)) {
/* 245 */       throw ServiceException.INVALID_REQUEST(sm.getClass().getSimpleName() + " is not supported", null);
/*     */     }
/*     */     
/* 248 */     this.mailboxId = mboxId;
/* 249 */     this.checkSize = checkSize;
/* 250 */     this.reportUsedBlobs = reportUsedBlobs;
/* 251 */     this.results = new Results();
/* 252 */     Mailbox mbox = MailboxManager.getInstance().getMailboxById(this.mailboxId);
/* 253 */     DbPool.DbConnection conn = null;
/*     */     try
/*     */     {
/* 256 */       conn = DbPool.getConnection();
/*     */       
/* 258 */       for (i$ = volumeIds.iterator(); i$.hasNext();) { short volumeId = ((Short)i$.next()).shortValue();
/* 259 */         Volume vol = VolumeManager.getInstance().getVolume(volumeId);
/* 260 */         if (vol.getType() == 10) {
/* 261 */           log.warn("Skipping index volume %d.  Only message volumes are supported.", new Object[] { Short.valueOf(vol.getId()) });
/*     */         }
/*     */         else {
/* 264 */           int numGroups = 1 << vol.getFileGroupBits();
/* 265 */           int filesPerGroup = 1 << vol.getFileBits();
/* 266 */           int mailboxMaxId = DbBlobConsistency.getMaxId(conn, mbox);
/*     */           
/*     */ 
/*     */ 
/*     */ 
/* 271 */           int minId = 0;
/* 272 */           int group = 0;
/* 273 */           int maxId = 0;
/* 274 */           while ((minId <= mailboxMaxId) && (group < numGroups))
/*     */           {
/*     */ 
/* 277 */             Multimap<Integer, BlobInfo> blobsById = HashMultimap.create();
/* 278 */             String blobDir = vol.getBlobDir(mbox.getId(), minId);
/*     */             
/* 280 */             while (minId <= mailboxMaxId) {
/* 281 */               maxId = minId + filesPerGroup - 1;
/* 282 */               for (BlobInfo blob : DbBlobConsistency.getBlobInfo(conn, mbox, minId, maxId, volumeId)) {
/* 283 */                 blobsById.put(Integer.valueOf(blob.itemId), blob);
/*     */               }
/* 285 */               minId += numGroups * filesPerGroup;
/*     */             }
/*     */             try {
/* 288 */               check(volumeId, blobDir, blobsById);
/*     */             } catch (IOException e) {
/* 290 */               throw ServiceException.FAILURE("Unable to check " + blobDir, e);
/*     */             }
/*     */             
/* 293 */             group++;
/* 294 */             minId = group * filesPerGroup;
/*     */           }
/*     */         }
/*     */       } } finally { Iterator i$;
/* 298 */       DbPool.quietClose(conn);
/*     */     }
/* 300 */     return this.results;
/*     */   }
/*     */   
/* 303 */   private static final Pattern PAT_BLOB_FILENAME = Pattern.compile("([0-9]+)-([0-9]+)\\.msg");
/*     */   
/*     */ 
/*     */ 
/*     */ 
/*     */   private void check(short volumeId, String blobDirPath, Multimap<Integer, BlobInfo> blobsById)
/*     */     throws IOException
/*     */   {
/* 311 */     Multimap<Integer, BlobInfo> revisions = HashMultimap.create();
/* 312 */     File blobDir = new File(blobDirPath);
/* 313 */     File[] files = blobDir.listFiles();
/* 314 */     if (files == null) {
/* 315 */       files = new File[0];
/*     */     }
/* 317 */     log.info("Comparing %d items to %d files in %s.", new Object[] { Integer.valueOf(blobsById.size()), Integer.valueOf(files.length), blobDirPath });
/* 318 */     for (File file : files)
/*     */     {
/* 320 */       Matcher matcher = PAT_BLOB_FILENAME.matcher(file.getName());
/* 321 */       int itemId = 0;
/* 322 */       int modContent = 0;
/* 323 */       if (matcher.matches()) {
/* 324 */         itemId = Integer.parseInt(matcher.group(1));
/* 325 */         modContent = Integer.parseInt(matcher.group(2));
/*     */       }
/*     */       
/* 328 */       BlobInfo blob = null;
/* 329 */       if (blobsById.containsKey(Integer.valueOf(itemId))) {
/* 330 */         Iterator<BlobInfo> iterator = blobsById.get(Integer.valueOf(itemId)).iterator();
/* 331 */         while (iterator.hasNext()) {
/* 332 */           BlobInfo tempBlob = (BlobInfo)iterator.next();
/* 333 */           if (tempBlob.modContent == modContent) {
/* 334 */             blob = tempBlob;
/* 335 */             revisions.put(Integer.valueOf(itemId), tempBlob);
/* 336 */             iterator.remove();
/*     */           }
/*     */         }
/*     */       }
/*     */       
/* 341 */       if (blob == null) {
/* 342 */         BlobInfo unexpected = new BlobInfo();
/* 343 */         unexpected.volumeId = volumeId;
/* 344 */         unexpected.path = file.getAbsolutePath();
/* 345 */         unexpected.fileSize = Long.valueOf(file.length());
/* 346 */         this.results.unexpectedBlobs.put(Integer.valueOf(itemId), unexpected);
/*     */       } else {
/* 348 */         blob.fileSize = Long.valueOf(file.length());
/* 349 */         blob.fileModContent = modContent;
/* 350 */         if (this.reportUsedBlobs) {
/* 351 */           this.results.usedBlobs.put(Integer.valueOf(blob.itemId), blob);
/*     */         }
/* 353 */         if (this.checkSize) {
/* 354 */           blob.fileDataSize = Long.valueOf(getDataSize(file, blob.dbSize));
/* 355 */           if (blob.dbSize != blob.fileDataSize.longValue()) {
/* 356 */             this.results.incorrectSize.put(Integer.valueOf(blob.itemId), blob);
/*     */           }
/*     */         }
/*     */       }
/*     */     }
/*     */     
/*     */ 
/* 363 */     for (BlobInfo blob : blobsById.values()) {
/* 364 */       this.results.missingBlobs.put(Integer.valueOf(blob.itemId), blob);
/*     */     }
/*     */     
/*     */ 
/*     */ 
/*     */ 
/* 370 */     Iterator<Integer> keyIterator = this.results.missingBlobs.keySet().iterator();
/* 371 */     while (keyIterator.hasNext()) {
/* 372 */       int itemId = ((Integer)keyIterator.next()).intValue();
/* 373 */       List<BlobInfo> missingBlobs = new ArrayList(this.results.missingBlobs.get(Integer.valueOf(itemId)));
/* 374 */       List<BlobInfo> unexpectedBlobs = new ArrayList(this.results.unexpectedBlobs.get(Integer.valueOf(itemId)));
/* 375 */       if ((missingBlobs.size() == 1) && (unexpectedBlobs.size() == 1) && (revisions.get(Integer.valueOf(itemId)).size() == 0)) {
/* 376 */         BlobInfo incorrectRevision = new BlobInfo();
/* 377 */         BlobInfo missingBlob = (BlobInfo)missingBlobs.get(0);
/* 378 */         incorrectRevision.itemId = missingBlob.itemId;
/* 379 */         incorrectRevision.modContent = missingBlob.modContent;
/* 380 */         incorrectRevision.dbSize = missingBlob.dbSize;
/* 381 */         incorrectRevision.volumeId = missingBlob.volumeId;
/*     */         
/* 383 */         BlobInfo unexpectedBlob = (BlobInfo)unexpectedBlobs.get(0);
/* 384 */         incorrectRevision.path = unexpectedBlob.path;
/* 385 */         incorrectRevision.fileSize = unexpectedBlob.fileSize;
/* 386 */         incorrectRevision.fileModContent = unexpectedBlob.fileModContent;
/*     */         
/* 388 */         this.results.incorrectModContent.put(Integer.valueOf(incorrectRevision.itemId), incorrectRevision);
/* 389 */         keyIterator.remove();
/* 390 */         this.results.unexpectedBlobs.removeAll(Integer.valueOf(itemId));
/*     */       }
/*     */     }
/*     */   }
/*     */   
/*     */   protected long getDataSize(File file, long expected) throws IOException
/*     */   {
/* 397 */     long fileLen = file.length();
/* 398 */     if ((fileLen != expected) && (FileUtil.isGzipped(file))) {
/* 399 */       return ByteUtil.getDataLength(new GZIPInputStream(new FileInputStream(file)));
/*     */     }
/* 401 */     return fileLen;
/*     */   }
/*     */   
/*     */   public static class BlobInfo
/*     */   {
/*     */     public int itemId;
/*     */     public int modContent;
/*     */     public int version;
/*     */     public long dbSize;
/*     */     public String path;
/*     */     public short volumeId;
/*     */     public int fileModContent;
/*     */     public Long fileDataSize;
/*     */     public Long fileSize;
/*     */     public boolean external;
/*     */     public IOException fetchException;
/*     */     public String locator;
/*     */   }
/*     */ }


/* Location:              /home/mint/zimbrastore.jar!/com/zimbra/cs/store/file/BlobConsistencyChecker.class
 * Java compiler version: 7 (51.0)
 * JD-Core Version:       0.7.1
 */